from pymongo import MongoClient
from static.configfile.config import *
from xpinyin import Pinyin
import jieba
import jieba.analyse as analyse
import time
import re
from static.configfile.config import *
# mongo_url_to_db = mongo_url_to_db
mongo_url_to_db = 'mongodb://yuqing:Bigdata#2021@175.27.160.162:27017'
analyse.set_stop_words("./stopwords.txt")
p = Pinyin()
stopwords = []
ping_list = { "weixin": "微信", "weibo": "微博", "baijiahao": "百家号" ,"baidu": "百度"}
with open("./stopwords.txt","r",encoding="utf-8") as f:
    stopwords = f.read().splitlines()
# 词云图
def word_cloud_data(collection,dbname):
    dataset = ""
    with collection.find({},{"_id":0,'mainText':1,'timestamp':1},no_cursor_timeout=True) as dr:
        for d in dr:
            dataset+=d["mainText"]
    dataset = re.sub(r"[\sa-zA-Zu2000-u206F%][\sa-zA-Zu2000-u206F%]+", "", dataset)
    keywords = analyse.extract_tags(dataset, topK=100, withWeight=True, allowPOS=())
    words = jieba.cut(dataset)
    feq = {}
    keywords = sorted(keywords, key = lambda kv:(kv[1], kv[0]))
    for word in words:
        feq[word] = feq.get(word,0)+1
    result = []
    for k in keywords:
        result.append({"name":k[0],"value":feq[k[0]]})
    return sorted(result,key=lambda x:x["value"],reverse=True)

# 发展趋势
def developmentchart(collection,dbname,taskname):
    # 转换成标准格式的年月日
    def to_ymd(x) -> str:
        return time.strftime("%Y-%m-%d",time.localtime(x*86400+28800))
    yarr = {}
    # for i in collection.aggregate([{"$project":{"_id":0,"id":1,"channel":1,"getTime":1,"ymd":{"$divide":[{"$subtract":[{"$add":["$getTime",28800]},{"$mod":[{"$add":["$getTime",28800]},86400]}]},86400]}}},{'$group': {'_id': {"channel":'$channel',"ymd":"$ymd"},'count': {'$sum': 1}}},{"$project":{"_id":0,"channel":"$_id.channel","ymd":"$_id.ymd","count":1}}]):
    for i in collection.aggregate([{"$project":{"_id":0,"id":1,"channel":1,"pubTime":1,"ymd":{"$divide":[{"$subtract":[{"$add":["$pubTime",28800]},{"$mod":[{"$add":["$pubTime",28800]},86400]}]},86400]}}},{'$group': {'_id': {"channel":'$channel',"ymd":"$ymd"},'count': {'$sum': 1}}},{"$project":{"_id":0,"channel":"$_id.channel","ymd":"$_id.ymd","count":1}}]):
        yarr.setdefault(i["channel"],{})
        yarr[i["channel"]][to_ymd(i["ymd"])] = i["count"]
    # 横坐标，时间轴
    xDataArr = []
    client = MongoClient(mongo_url_to_db)
    task_collection = client["Task"]["task"]
    with task_collection.find({"name":taskname},{'_id':0,"canshu":1,"savetime":1,'timestamp':1},no_cursor_timeout=True) as ret:
            for r in ret:
                fromdate = r["canshu"]["fromdate"]
                savedate = r["savetime"]
    from datetime import datetime,timedelta
    f = [int(y) for y in fromdate.split("-")]
    t = [int(y) for y in savedate.split("-")]
    td = datetime(t[0], t[1], t[2])
    fd = datetime(f[0], f[1], f[2])
    Days = (td-fd).days
    xDataArr.append(fd.strftime("%Y-%m-%d"))
    for i in range(Days):
        fd=fd+timedelta(days=+1)
        xDataArr.append(fd.strftime("%Y-%m-%d"))

    # 纵坐标 每个平台数据量
    legend = list(yarr.keys())
    series =[]
    for l in legend:
        temp = yarr[l]
        data = []
        for d in xDataArr:
            data.append(temp.setdefault(d,0))
        y = {
                "type": 'line',
                "areaStyle":{},
                "stack": 'Total'
            }
        y["name"] = ping_list[l]
        y["data"] = data
        series.append(y)
    chartdata = {}
    chartdata["xDataArr"] = xDataArr
    chartdata["series"] = series
    chartdata["legend"] = [ping_list[i] for i in legend]
    # 事件概括文案
    text_data = {}
    # 每个平台的数据数量
    ping_data_list = []
    for k in yarr.keys():
        ping_data_list.append((ping_list[k],sum(yarr[k].values())))
    ping_data_list = sorted(ping_data_list,key=lambda x:(x[1],x[0]),reverse=True)
    text_data["max"] = ping_data_list[0]
    text_data["fromdate"] = "{0[0]}年{0[1]}月{0[2]}日".format(fromdate.split("-"))
    text_data["savedate"] = "{0[0]}年{0[1]}月{0[2]}日".format(savedate.split("-"))
    text_data["all"] = collection.count_documents({})
    with collection.find({},{"_id":0,"author":1,"pubTime":1,"channel":1,"timestamp":1}).sort("pubTime",1).limit(1) as ret:
        for r in ret:
            # print(type(time.localtime(86400+28800)))
            # print(r["pubTime"])
            # print(time.strftime("%Y-%m-%d-%H-%M-%S",r["pubTime"]))
            text_data["first"] = "{0[0]}年{0[1]}月{0[2]}日{0[3]}:{0[4]}".format(time.strftime("%Y-%m-%d-%H-%M",time.localtime(r["pubTime"])).split("-"))
            text_data["first-pingtai"] = ping_list[r["channel"]]
    chartdata["text_data"] = text_data
    return chartdata


# 地理位置
def map(collection,dbname):
    data = {}
    exclude = ["海外","其他","其它"]
    with collection.find({"location":{"$regex":"\W"}},{"_id":0,"location":1,"timestamp":1},no_cursor_timeout=True) as f:
        for i in f:
            location = re.split("·| |-", i["location"])[0]
            if location not in exclude:
                data[location] = data.get(location,0)+1
    result = []
    # {"name": '澳门', "value": 147}
    for k,v in data.items():
        result.append({"name": k, "value": v})
    
    return sorted(result,key=lambda x:x["value"],reverse=True)

# 用户认证数据
def user(collection,dbname):
    '''
        [
            { value: 1048, name: 'Search Engine' },
            { value: 735, name: 'Direct' },
            { value: 580, name: 'Email' },
            { value: 484, name: 'Union Ads' },
            { value: 300, name: 'Video Ads' }
          ]
    '''
    data = []
    all_num = collection.count_documents({})
    print(all_num)
    # 黄v
    orange_number = collection.count_documents({"IsVerified":True,"verified_type":0})
    print(orange_number)
    data.append({"value":orange_number,"name":"黄V"})
    # 蓝v
    blue_number = collection.count_documents({"IsVerified":True,"verified_type":{"$gt":0}})
    print(blue_number)
    data.append({"value":blue_number,"name":"蓝V"})
    normal_number = all_num-blue_number-orange_number
    data.append({"value":normal_number,"name":"普通用户"})
    blue_data = {}
    temp_data = {"10":{},"0":{},"1":{},"2":{},"3":{},"9":{}}
    
    fans_data = {}
    with collection.find({},{"_id":0,"author":1,"IsVerified":1,"fansNumber":1,"verified_type":1,"timestamp":1},no_cursor_timeout=True) as ret:
        for r in ret:
            # # {名字：{发文数:，粉丝数:，认证团体类型:}}
            # if r["author"] not in blue_data.keys():
            #     blue_data[r["author"]] = {"pub":1,"fans":0,"type":r["verified_type"]}
            # else:
            #     blue_data[r["author"]]["fans"] += 1
            #     if "fansNumber" in r.keys():
            #         blue_data[r["author"]]["fans"] = r["fansNumber"]

            # {类型：{昵称：{发文数，粉丝数}}}
            if "IsVerified" not in r.keys() or r["IsVerified"] == False:
                r["verified_type"]  = "10"
            elif r["verified_type"] > 3:
                r["verified_type"]  = "9"
            else: r["verified_type"] = str(r["verified_type"])
            if r["author"] not in temp_data[r["verified_type"]].keys():
                temp_data[r["verified_type"]][r["author"]] = {"pub":1,"fans":0}
            else:
                temp_data[r["verified_type"]][r["author"]]["pub"] += 1
                if "fansNumber" in r.keys():
                    temp_data[r["verified_type"]][r["author"]]["fans"] = r["fansNumber"]
    # print(temp_data)
    for k in temp_data.keys():
        tttt = sorted(temp_data[k].items(),key=lambda x:x[1]["fans"],reverse=True)
        temp_data[k] = tttt[:min(len(tttt), 12)] if len(tttt)>0 else []
    # return {"chartData":data,"tabledata":sorted(blue_data.items(),key=lambda x:x[1]["fans"],reverse=True)}
    return {"chartData":data,"tabledata":temp_data}

# 微博类型
def typeofweibo(collection,dbname,taskname):
# ------------饼状图数据----------------
    allnum = collection.count_documents({})
    Retweets = collection.count_documents({"type":"WeiboRetweets"})
    Comments = collection.count_documents({"type":"WeiboComments"})
    original = collection.count_documents({"type":"original"})
    piedata = []
    piedata.append({"value":Comments,"name":"评论"})
    piedata.append({"value":original,"name":"原创"})
    piedata.append({"value":Retweets,"name":"转发"})

# ----------折线图数据--------------
    type_list = { "original": "原创", "WeiboRetweets": "转发", "WeiboComments": "评论"}
    # 转换成标准格式的年月日
    def to_ymd(x) -> str:
        return time.strftime("%Y-%m-%d",time.localtime(x*86400+28800))
    yarr = {}
    # for i in collection.aggregate([{"$project":{"_id":0,"id":1,"channel":1,"getTime":1,"ymd":{"$divide":[{"$subtract":[{"$add":["$getTime",28800]},{"$mod":[{"$add":["$getTime",28800]},86400]}]},86400]}}},{'$group': {'_id': {"channel":'$channel',"ymd":"$ymd"},'count': {'$sum': 1}}},{"$project":{"_id":0,"channel":"$_id.channel","ymd":"$_id.ymd","count":1}}]):
    for i in collection.aggregate([{"$project":{"_id":0,"id":1,"type":1,"pubTime":1,"ymd":{"$divide":[{"$subtract":[{"$add":["$pubTime",28800]},{"$mod":[{"$add":["$pubTime",28800]},86400]}]},86400]}}},{'$group': {'_id': {"type":'$type',"ymd":"$ymd"},'count': {'$sum': 1}}},{"$project":{"_id":0,"type":"$_id.type","ymd":"$_id.ymd","count":1}}]):
        if "type" not in i.keys():
            i["type"] = "original"
        yarr.setdefault(i["type"],{})
        yarr[i["type"]][to_ymd(i["ymd"])] = i["count"]
    # 横坐标，时间轴
    xDataArr = []
    client = MongoClient(mongo_url_to_db)
    task_collection = client["Task"]["task"]
    with task_collection.find({"name":taskname},{'_id':0,"canshu":1,"savetime":1,'timestamp':1},no_cursor_timeout=True) as ret:
            for r in ret:
                fromdate = r["canshu"]["fromdate"]
                savedate = r["savetime"]
    from datetime import datetime,timedelta
    f = [int(y) for y in fromdate.split("-")]
    t = [int(y) for y in savedate.split("-")]
    td = datetime(t[0], t[1], t[2])
    fd = datetime(f[0], f[1], f[2])
    Days = (td-fd).days
    xDataArr.append(fd.strftime("%Y-%m-%d"))
    for i in range(Days):
        fd=fd+timedelta(days=+1)
        xDataArr.append(fd.strftime("%Y-%m-%d"))

    # 纵坐标 每个平台数据量
    legend = list(yarr.keys())
    series =[]
    temp_data = []
    for l in legend:
        temp = yarr[l]
        data = []
        for d in xDataArr:
            data.append(temp.setdefault(d,0))
        y = {
                "type": 'line',
                "areaStyle":{},
                "stack": 'Total'
            }
        y["name"] = type_list[l]
        y["data"] = data
        temp_data.append(data)
        series.append(y)
    chartdata = {}
    chartdata["xDataArr"] = xDataArr
    chartdata["series"] = series
    chartdata["legend"] = [type_list[i] for i in legend]

# 文本数据
    textdata = {}
    textdata["all"] = allnum
    textdata["original"] = round(original*100/allnum,1)
    textdata["Retweets"] = round(Retweets*100/allnum,1)
    textdata["Comments"] = round(100 - textdata["original"] - textdata["Retweets"],1)
    max_index = 0
    max_num = -1
    for i in range(Days):
        temp_max = 0
        for j in range(len(temp_data)):
            temp_max+=temp_data[j][i]
        if temp_max > max_num:
            max_num = temp_max
            max_index = i
    textdata["max"] = max_num
    textdata["max_index"] = "{0[0]}年{0[1]}月{0[2]}日".format(xDataArr[max_index].split("-"))

    respone = {}
    respone["piedata"] = piedata
    respone["chartdata"] = chartdata
    respone["textdata"] = textdata
    return respone

def main(collection,dbname,taskname):
    respone = {}
    respone["name"] = dbname
    try:
        respone["word_cloud_data"] = word_cloud_data(collection, dbname)
        print("word_cloud_data")
    except:
        print("词云图数据出错")
    
    try:
        respone["developmentchart"] = developmentchart(collection, dbname, taskname)
        print("developmentchart")
    except:
        print("发展趋势折线图出错")

    try:
        respone["map"] = map(collection, dbname)
        print("map")
    except:
        print("地图出错")
    
    try:
        respone["user"] = user(collection, dbname)
        print("user")
    except:
        print("用户类型数据出错")
    
    try:
        respone["typeofweibo"] = typeofweibo(collection, dbname, taskname)
        print("typeofweibo")
    except:
        print("微博类型数据出错")
    client = MongoClient(mongo_url_to_db)
    EventData_collection = client["Task"]["EventData"]
    if not EventData_collection.find_one({"name":dbname}):
        EventData_collection.insert_one(respone)
        print("插入")
    else:
        EventData_collection.update_one({"name":dbname}, {"$set":respone})
        print("更新")

if __name__ == "__main__":
    List = ["俄乌局势"]
    for l in List:
        taskname = l
        dbname = "doc_"+str(p.get_pinyin(taskname,""))
        client =MongoClient(mongo_url_to_db)
        collection =client["Task"][dbname]
        main(collection, dbname, taskname)
    

